#!/usr/bin/python
# -*- coding: utf-8 -*-

"""
PErsonal Python Enhancements library (general purpose and data-mining)

Copyright (C) 2010, "Fatura, Karel" <kfatura@deloitteCE.com>
Copyright (C) 2007-2009, "Viktor Presovsky" <vpresovsky@deloitteCE.com>
Copyright (C) 2003-2007, "Viktor Presovsky" <presovsky@gmail.com>
Copyright (C) 2009, "Michal Sagat" <msagat@deloitteCE.com>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.
"""

""" 
general soubroutines file 
last updated : 2010-09-16_0031, by vip
"""

import glob
import os
import re
import string
import sys
import time
import shutil
import platform

# posix 2.4, nt 2.4
OSPYVER = "%s %s" % (os.name, platform.python_version()[:-2])
#print OSPYVER

import pepe.lambdas as lamb
#import pepe.p_dm.dama as dama
import pepe.p_dm.sdb as s
#import pepe.p_sql.pg as pg
import pepe.p_txt.bigtxt as bt
import pepe.p_txt.txt_columns as tc
import pepe.p_utils.csv_sas as csvs
import pepe.p_utils.srchr as srchr
#import pepe.p_utils.txt2pg as ppt
import pepe.p_utils.txtr as p_txtr
import pepe.p_utils.uid_uuid as uuid

zf9 = lambda x: string.zfill(str(x), 9)
# 2683099 -> *002.683.099*
x3x = lambda x: "*%s.%s.%s*" % (zf9(x)[:3], zf9(x)[3:6], zf9(x)[6:9])

fw_slash = lambda x: re.sub(r'\\', '/', x)

#print lamb.dtnow() # datetime stamp
timenow = lambda : time.strftime ("%Y%m%dT%H%M%S", time.localtime())
#print timenow()


def file_hash(ifile, type='sha256'):
    ''' calculate checksum for file
    '''
    import pepe.p_filesystem.dfstats as dfs
    
    rd = dfs.dfstats(ifile, options=[type], pprint=False)
    for f in sort_case_insensitive(rd.keys()):
        x = "%s *%s\n" % (rd[f][type], rd[f]['file_name'])
        return x

def strip_txt_columns(txt,cd=''):
    ''' '''
    line = re.sub('\n', '', txt)
    cols = string.split(line, cd)
    bare_cols = [string.strip(c) for c in cols]
    return string.join(bare_cols, cd) + '\n'

def sort_case_insensitive(stringList):
    ''' 
    http://code.activestate.com/recipes/170242/
    
    The default compare function used when sorting a list of strings uses the ordinal value 
    of the string characters for comparison. This results in the typical ASCII sort result 
    of 'B' < 'a'. If a user is going to see the list, it is generally better to do a case-insensitive comparison.
	
	Note: doesn't do locale-specific compare

    L = ['bank_ucty.txt', 'chs.dat#', 'dodav_pop.txt', 'dr_dokladu.txt', 'polozky.7z', 'uc_klice.txt', 'Dodavatele.txt', 'SR71.xls']
    
    print sorted(L)
    ['Dodavatele.txt', 'SR71.xls', 'bank_ucty.txt', 'chs.dat#', 'dodav_pop.txt', 'dr_dokladu.txt', 'polozky.7z', 'uc_klice.txt']
    
    print sort_case_insensitive(L)
    ['bank_ucty.txt', 'chs.dat#', 'dodav_pop.txt', 'Dodavatele.txt', 'dr_dokladu.txt', 'polozky.7z', 'SR71.xls', 'uc_klice.txt']
    '''
    tupleList = [(x.upper(), x) for x in stringList]
    tupleList.sort()
    return [x[1] for x in tupleList]

def findcoldelimiter_line(line):
    ''' guess what text file column delimiter is used 

    if cd == '': cd = findcoldelimiter_line(line)

    >>> findcoldelimiter_line("a|^b;^c~^d~")
    '^'
    '''
    c = 0; r = '' 
    for cs in [',', '\t', ';', '^', '~', '|']:
        if string.count(line, cs) > c:
            c = string.count(line, cs); r = cs
    return r

def findcoldelimiter(line):
    ''' maintained for backward compatibility purposes 
        use findcoldelimiter_line() in new projects
    '''
    return findcoldelimiter_line(line)

def findcoldelimiter_file(inputfile):
    ''' '''
    line = open(inputfile, 'r').readline()
    return findcoldelimiter_line(line)

def lol_column(L, i):
    ''' return column from list-of-lists 
        L - list of lists
        i - column index
    >>> lol_column([[1,2],[3,4]], 1)
    [2, 4]
    '''
    return [x[i] for x in L]

"""    
def oldString2number(x):

    # handle following in some utf8 files ... 65Â 000,00
    x = re.sub('\xC2\xA0','', x)
    x = re.sub('Â ','', x)

    res = 0
    x = re.sub('[+ěĚšŠčČřŘžŽýÝáÁíÍéÉóÓúÚ a-zA-Z_]', '', x)

    # dot or coma : .., or ,,.
    doc = re.sub('(-| |\d)', '', x)
    if x in ('-', '0-', '-0', '.', ','): x = ''
    if len(x) == 0:
        return 0
    else:
        if len(doc) <> 0:
            # case 1.234,45-
            if doc[-1] == ',': res = float(re.sub(',', '.', re.sub('(\.|-)', '', x)))
            # case 1,234.45-
            if doc[-1] == '.': res = float(re.sub('(,|-)', '', x))
        else:
            res = float(x)
        if string.find(x, '-') <> -1: res = res*-1
        return res
"""
   
def string2number(x):
    ''' convert string to number 

    >>> string2number('51.12919')
    (51.12919)

    >>> string2number(';'), string2number(':')
    (0.0, 0.0)

    >>> string2number('**'), string2number('***')
    (0.0, 0.0)

    >>> string2number('--'), string2number('---')
    (0.0, 0.0)

    >>> string2number('íáýsas,msmsas _'), string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě')
    (0.0, 0.0)

    >>> string2number(''), string2number('-'), string2number('-0'), string2number('0-')
    (0.0, 0.0, 0.0, 0.0)

    >>> string2number('9.991.234,45-'), string2number('9,991,234.45-')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45-'), string2number('-9991234.45-')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45'), string2number('-9991234.45')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('íáýsas,msmsas _'), string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě')
    (0.0, 0.0)

    >>> string2number(''), string2number('-'), string2number('-0'), string2number('0-')
    (0.0, 0.0, 0.0, 0.0)

    >>> string2number('9.991.234,45-'), string2number('9,991,234.45-')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45-'), string2number('-9991234.45-')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45'), string2number('-9991234.45')
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě'), string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě')
    (0.0, 0.0)

    >>> string2number('9.991.234,45-'),  string2number('9,991,234.45-'), string2number('-9991234,45-')
    (-9991234.4499999993, -9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234.45-'), string2number('-9991234,45'), string2number('-9991234.45')
    (-9991234.4499999993, -9991234.4499999993, -9991234.4499999993)

    >>> string2number('12345'), string2number('-12,345'), string2number('-12 345')
    (12345.0, -12345.0, -12345.0)

    >>> string2number('-12 345.00'),string2number('-12,34'), string2number('123 456')
    (-12345.0, -12.34, 123456.0)

    >>> string2number('234'), string2number('2-'), string2number('23,-')
    (234.0, -2.0, 23.0)

    >>> string2number('-2'), string2number(',45'), string2number('1.234.567')
    (-2.0, 0.45000000000000001, 1234567.0)

    >>> string2number('1,234,567'), string2number('-1,234,567'), string2number('-1.234.567')
    (1234567.0, -1234567.0, -1234567.0)

    >>> string2number('65 000,00') # this one has NON-BRAKE SPACE
    65000.0

    >>> string2number('1,234,567-')
    -1234567.0
    
    >>> string2number('íáýsas,msmsas _'), string2number('wewšěě+š+ě+ěwewšěě+š+ě+ě') #a
    (0.0, 0.0)

    >>> string2number(''), string2number('-'), string2number('-0'), string2number('0-') #b
    (0.0, 0.0, 0.0, 0.0)

    >>> string2number('9.991.234,45-'), string2number('9,991,234.45-') #c
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45-'), string2number('-9991234.45-') #d
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number('-9991234,45'), string2number('-9991234.45') #e
    (-9991234.4499999993, -9991234.4499999993)

    >>> string2number("íáýsas,msmsas _"), string2number("wewšěě+š+ě+ěwewšěě+š+ě+ě") #f
    (0.0, 0.0)
    
    >>> string2number(''), string2number("-0"), string2number("0-") #g
    (0.0, 0.0, 0.0)

    >>> string2number("9.991.234,45-") #h
    -9991234.4499999993

    >>> string2number("-9991234,45-") #i
    -9991234.4499999993

    >>> string2number("-9991234.45-") #j
    -9991234.4499999993

    >>> string2number("-9991234,45") #k
    -9991234.4499999993

    >>> string2number("-9991234.45") #l
    -9991234.4499999993

    >>> string2number("12345") #m
    12345.0

    >>> string2number("-12,345") #n
    -12345.0

    >>> string2number("9.991.234,45-") #h
    -9991234.4499999993

    >>> string2number("9.991.234,45-") #h
    -9991234.4499999993

    >>> string2number("9.991.234,45-") #h
    -9991234.4499999993

    >>> string2number('65Â 000,00')
    65000.0
    
    >>> string2number('(USD 30,420.00-)')
    -30420.0
    '''

    # 1. clearing the given value
    x = x.strip()
    y = x
    x = re.sub('\xC2\xA0','', x)                                # NON-BRAKE SPACE
    x = re.sub('Â| ','', x)                                     # handle following in some utf8 files ... 65Â 000,00
    x = re.sub('[*:;()+ěĚšŠčČřŘžŽýÝáÁíÍéÉóÓúÚ a-zA-Z_]', '', x)   # other unwanted characters
                                                                 # 2010-09-14_1155 vp added  ()
                                                                 # 2010-10-11_2259 vp added  *:;
    x = re.sub('-{2,}', '', x)                                  # 2010-10-11_1635 vp ... if more than one, its not an minus
    
    if x == '' or x == '-':
        return float(0)                                         # EXIT of: non numeric values free of . , -
    
    # 2. sign + or -?
    sign = 1
    if x[0] == '-' or re.match('\d-',x[-2:]):                   # -123 or 123- or 123,45- or -123,- but not 123,-
        sign = -1
    x = re.sub('-','',x)                                        # x does not need negative indicator any more

    # 3. number format interpretation
    # dot or coma : .., or ,,.
    doc = re.sub('(-|\d)', '', x)
    if doc == '':
        return float(x) * sign                                  # EXIT of: not formated integers
    else:
        sameDelims = True                                       # are all the delimiters the same?
        prevCh = doc[0]
        for currCh in doc:
            if currCh != prevCh:
                sameDelims = False
            prevCh = currCh
    if sameDelims:
        if re.match('\d{3}',x[-3:]):                            # is x like 123.456 - thousends delimiter or
            x = re.sub(',|\.','',x)                             # 123.45 - floating point?
        elif len(doc) == 1:
            x = re.sub(',','.',x)
        else:                                                   # i dont understand for example this 1.234.56
            raise 'could not recognize number format of: ',y
    else:
        if doc[-1] == '.':                                      # if there is a floating point, thousands delimiter
            x = re.sub(',','',x)                                # can be deleted
        elif doc[-1] == ',':
            x = re.sub('\.','',x)                               # for czech format '.'s needs to be dropped and the
            x = re.sub(',','.',x)                               # last ',' is converted to floating point
        else:
            raise 'could not recognize number format of: ',y
    x = '0' + x                                                 # in the clarify proces something like '.' can
    return float(x) * sign                                      # happend - but zero on the begining soves the problem
    # EXIT of: all others here                                  # without changing the value. (+- are already droped)


if __name__== '__main__':
    """ """
    import doctest
    doctest.testmod()

